# encoding: utf-8

import json

path = "C:/Users/Administrator/Downloads/chinese_enthusiastic_dpo.jsonl"

with open(path, "r", encoding="utf-8") as f:
    with open("./datas/dpo.json", "w", encoding="utf-8") as ff:
        for line in f:
            data = json.loads(line)
            chosen = data.get("chosen", "")
            rejected = data.get("rejected", "")
            if chosen == rejected:
                continue
            ff.write(json.dumps(data, ensure_ascii=False)+"\n")
